﻿using System;
using System.Collections.Generic;
using System.Windows.Forms;
using System.Net.Http;
using System.Text.RegularExpressions;

namespace Assignment7
{
    public partial class Form2 : Form
    {
        private readonly List<string> urlsToCrawl = new List<string>();
        // 已经爬取的URL
        private readonly HashSet<string> crawledUrls = new HashSet<string>();
        // 错误的URL
        private readonly List<string> errorUrls = new List<string>();
        private string baseUri;
        public Form2()
        {
            InitializeComponent();
            
        }

        private void Form2_Load(object sender, EventArgs e)
        {
            button1.Click += StartCrawl_Click;
        }
        private async void StartCrawl_Click(object sender, EventArgs e)
        {
            baseUri = textBox1.Text.Trim();
            int cnt = 0;

            // 如果不是以http://或https://开头，则补上协议
            if (!Regex.Match(baseUri, "^https?://").Success)
            {
                baseUri = "http://" + baseUri;
            }
            urlsToCrawl.Add(baseUri);

            while (urlsToCrawl.Count > 0 && cnt < 50)
            {
                string url = urlsToCrawl[0];
                urlsToCrawl.RemoveAt(0);

                if (crawledUrls.Contains(url))
                {
                    continue;
                }

                var http = new HttpClient();
                try
                {
                    Console.WriteLine("正在爬取 {0}", url);
                    var response = await http.GetAsync(url);
                    response.EnsureSuccessStatusCode();
                    var html = await response.Content.ReadAsStringAsync();
                    Console.WriteLine("页面HTML内容：\r\n{0}", html);

                    // 记录已经爬取的URL
                    crawledUrls.Add(url);
                    listBox1.Items.Add(url);

                    // 使用HtmlAgilityPack解析HTML文档
                    var doc = new HtmlAgilityPack.HtmlDocument();
                    doc.LoadHtml(html);

                    // 遍历链接，将相对地址转成完整地址并判断是否是符合要求的网页
                    foreach (var link in doc.DocumentNode.Descendants("a"))
                    {
                        string href = link.GetAttributeValue("href", null);
                        if (!string.IsNullOrWhiteSpace(href))
                        {
                            // 将相对地址转成完整地址
                            Uri currentUri = new Uri(baseUri);
                            Uri hrefUri;
                            if (Uri.TryCreate(href, UriKind.Relative, out hrefUri))
                            {
                                Uri absoluteUri = new Uri(currentUri, hrefUri);
                                href = absoluteUri.ToString();
                            }

                            // 判断是否是符合要求的网页
                            if (IsValidUrl(href))
                            {
                                urlsToCrawl.Add(href);
                            }
                            else
                            {
                                errorUrls.Add(href);
                                listBox1.Items.Add("[错误] " + href);
                            }
                        }
                    }

                    cnt++;
                }
                catch
               (Exception ex)
                {
                    errorUrls.Add(url);
                    listBox1.Items.Add("[] " + url + "：" + ex.Message);
                }
            }

            MessageBox.Show(string.Format("爬取结束，共爬取 {0} 个URL，{1} 个错误", crawledUrls.Count, errorUrls.Count));
        }

        private static bool IsValidUrl(string url)
        {
            string pattern = @".*\.(htm|html|aspx|php|jsp)$";
            return Regex.IsMatch(url, pattern, RegexOptions.IgnoreCase);
        }

        private void textBox1_TextChanged(object sender, EventArgs e)
        {

        }

        private void button1_Click(object sender, EventArgs e)
        {
            
        }

        private void listBox1_SelectedIndexChanged(object sender, EventArgs e)
        {

        }
    }
}


